{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Heavily borrowed from https://www.kaggle.com/jsaguiar/updated-0-792-lb-lightgbm-with-simple-features/code\n",
    "\n",
    "# Changes made 8/22: \n",
    "#   Added random forest model\n",
    "#   Function to replace NA's with medians and mark occurrences in new columns\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "import time\n",
    "from contextlib import contextmanager\n",
    "import lightgbm as lgb\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "from sklearn.model_selection import KFold, StratifiedKFold, train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "from skopt.space import Real, Integer\n",
    "from skopt.utils import use_named_args\n",
    "import itertools\n",
    "from skopt import gp_minimize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "@contextmanager\n",
    "def timer(title):\n",
    "    t0 = time.time()\n",
    "    yield\n",
    "    print(\"{} - done in {:.0f}s\".format(title, time.time() - t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# One-hot encoding for categorical columns with get_dummies\n",
    "def one_hot_encoder(df, nan_as_category = True):\n",
    "    original_columns = list(df.columns)\n",
    "    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "    df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)\n",
    "    new_columns = [c for c in df.columns if c not in original_columns]\n",
    "    return df, new_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocess application_train.csv and application_test.csv\n",
    "def application_train_test(num_rows = None, nan_as_category = False):\n",
    "    # Read data and merge\n",
    "    df = pd.read_csv('../data/application_train.csv', nrows= num_rows)\n",
    "    test_df = pd.read_csv('../data/application_test.csv', nrows= num_rows)\n",
    "    print(\"Train samples: {}, test samples: {}\".format(len(df), len(test_df)))\n",
    "    df = df.append(test_df).reset_index()\n",
    "    # Optional: Remove 4 applications with XNA CODE_GENDER (train set)\n",
    "    #df = df[df['CODE_GENDER'] != 'XNA']\n",
    "    \n",
    "    # Categorical features with Binary encode (0 or 1; two categories)\n",
    "    for bin_feature in ['CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY']:\n",
    "        df[bin_feature], uniques = pd.factorize(df[bin_feature])\n",
    "    # Categorical features with One-Hot encode\n",
    "    df, cat_cols = one_hot_encoder(df, nan_as_category)\n",
    "    \n",
    "    # NaN values for DAYS_EMPLOYED: 365.243 -> nan\n",
    "    df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)\n",
    "    # Some simple new features (percentages)\n",
    "    df['DAYS_EMPLOYED_PERC'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']\n",
    "    df['INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']\n",
    "    df['INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']\n",
    "    df['ANNUITY_INCOME_PERC'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']\n",
    "    df['PAYMENT_RATE'] = df['AMT_ANNUITY'] / df['AMT_CREDIT']\n",
    "    df['INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']\n",
    "    \n",
    "    df['CREDIT_INCOME_PERC'] = df['AMT_CREDIT'] / df['AMT_INCOME_TOTAL']\n",
    "    df['INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']\n",
    "    df['REGION_INCOME_RATIO'] = df['AMT_INCOME_TOTAL'] /  df['REGION_POPULATION_RELATIVE']\n",
    "    df['ANNUITY_INCOME_PERC'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']\n",
    "    df['GOODS_TO_INCOME_RATIO'] = df['AMT_GOODS_PRICE'] / df['AMT_INCOME_TOTAL']\n",
    "    df['ANNUITY_LENGTH'] = df['AMT_CREDIT'] / df['AMT_ANNUITY']\n",
    "    df['CHILDREN_RATIO'] = df['CNT_CHILDREN'] / df['CNT_FAM_MEMBERS'] \n",
    "    df['CREDIT_TO_GOODS_RATIO'] = df['AMT_CREDIT'] / df['AMT_GOODS_PRICE']\n",
    "    df['INC_PER_CHLD'] = df['AMT_INCOME_TOTAL'] / (1 + df['CNT_CHILDREN'])\n",
    "    df['SOURCES_PROD'] = df['EXT_SOURCE_1'] * df['EXT_SOURCE_2'] * df['EXT_SOURCE_3']\n",
    "    \n",
    "    df['CAR_TO_BIRTH_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_BIRTH']\n",
    "    df['CAR_TO_EMPLOY_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_EMPLOYED']\n",
    "\n",
    "    df['PHONE_TO_BIRTH_RATIO'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_BIRTH']\n",
    "    df['PHONE_TO_EMPLOY_RATIO'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_EMPLOYED']\n",
    "\n",
    "    df['FAM_MEMBERS_TO_LIVINGAREA_AVG'] = df['CNT_FAM_MEMBERS'] / df['LIVINGAREA_AVG']\n",
    "    df['FAM_MEMBERS_TO_NONLIVINGAREA_AVG'] = df['CNT_FAM_MEMBERS'] / df['NONLIVINGAREA_AVG']\n",
    "\n",
    "    df['FAM_MEMBERS_TO_AGE'] = df['CNT_FAM_MEMBERS'] / df['DAYS_BIRTH']\n",
    "    df['FAM_MEMBERS_TO_EXT_SOURCE_3'] = df['CNT_FAM_MEMBERS'] / df['EXT_SOURCE_3']\n",
    "       \n",
    "    df['CREDIT_TO_LIVINGAREA'] = df['AMT_CREDIT'] / df['LIVINGAREA_AVG']\n",
    "    df['CREDIT_TO_LANDAREA'] =  df['AMT_CREDIT'] / df['LANDAREA_AVG']\n",
    "    df['CREDIT_TO_TOTALAREA'] = df['AMT_CREDIT'] / df['TOTALAREA_MODE']\n",
    "    df['CREDIT_TO_BASEMENTAREA'] = df['AMT_CREDIT'] / df['BASEMENTAREA_AVG']\n",
    "    df['CREDIT_TO_EMPLOYED'] = df['AMT_CREDIT'] / df['DAYS_EMPLOYED']\n",
    "    df['CREDIT_TO_FAM_MEMBERS'] = df['AMT_CREDIT'] / df['CNT_FAM_MEMBERS']\n",
    "    df['CREDIT_TO_CHILDREN'] = df['AMT_CREDIT'] / (1 + df['CNT_CHILDREN'])\n",
    "    df['CREDIT_TO_EXT_SOURCE_3'] = df['AMT_CREDIT'] / df['EXT_SOURCE_3']\n",
    "    df['CREDIT_TO_EXT_SOURCE_2'] = df['AMT_CREDIT'] / df['EXT_SOURCE_2']\n",
    "    df['CREDIT_TO_EXT_SOURCE_1'] = df['AMT_CREDIT'] / df['EXT_SOURCE_1']\n",
    "\n",
    "    df['ANNUITY_TO_LIVINGAREA'] = df['AMT_ANNUITY'] / df['LIVINGAREA_AVG']\n",
    "    df['ANNUITY_TO_LANDAREA'] =  df['AMT_ANNUITY'] / df['LANDAREA_AVG']\n",
    "    df['ANNUITY_TO_TOTALAREA'] = df['AMT_ANNUITY'] / df['TOTALAREA_MODE']\n",
    "    df['ANNUITY_TO_BASEMENTAREA'] = df['AMT_ANNUITY'] / df['BASEMENTAREA_AVG']\n",
    "    df['ANNUITY_TO_EMPLOYED'] = df['AMT_ANNUITY'] / df['DAYS_EMPLOYED']\n",
    "    df['ANNUITY_TO_FAM_MEMBERS'] = df['AMT_ANNUITY'] / df['CNT_FAM_MEMBERS']\n",
    "    df['ANNUITY_TO_CHILDREN'] = df['AMT_ANNUITY'] / (1 + df['CNT_CHILDREN'])\n",
    "\n",
    "    df['EXT_SOURCE_3_REGION'] = df['EXT_SOURCE_3'] / df['REGION_RATING_CLIENT']\n",
    "    df['EXT_SOURCE_3_REGION_CITY'] = df['EXT_SOURCE_3'] / df['REGION_RATING_CLIENT_W_CITY']\n",
    "    df['EXT_SOURCE_2_REGION'] = df['EXT_SOURCE_2'] / df['REGION_RATING_CLIENT']\n",
    "    df['EXT_SOURCE_2_REGION_CITY'] = df['EXT_SOURCE_2'] / df['REGION_RATING_CLIENT_W_CITY']\n",
    "    df['EXT_SOURCE_1_REGION'] = df['EXT_SOURCE_1'] / df['REGION_RATING_CLIENT']\n",
    "    df['EXT_SOURCE_1_REGION_CITY'] = df['EXT_SOURCE_1'] / df['REGION_RATING_CLIENT_W_CITY']\n",
    "\n",
    "    df['INCOME_TO_EXT_SOURCE_3'] = df['AMT_INCOME_TOTAL'] / df['EXT_SOURCE_3']\n",
    "    df['INCOME_TO_EXT_SOURCE_2'] = df['AMT_INCOME_TOTAL'] / df['EXT_SOURCE_2']\n",
    "    df['INCOME_TO_EXT_SOURCE_1'] = df['AMT_INCOME_TOTAL'] / df['EXT_SOURCE_1']\n",
    "    df['INCOME_TO_REGION'] = df['AMT_INCOME_TOTAL'] / df['REGION_RATING_CLIENT']\n",
    "    df['INCOME_TO_REGION_CITY'] = df['AMT_INCOME_TOTAL'] / df['REGION_RATING_CLIENT_W_CITY']\n",
    "\n",
    "    del test_df\n",
    "    gc.collect()\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocess bureau.csv and bureau_balance.csv\n",
    "def bureau_and_balance(num_rows = None, nan_as_category = True):\n",
    "    bureau = pd.read_csv('../data/bureau.csv', nrows = num_rows)\n",
    "    bb = pd.read_csv('../data/bureau_balance.csv', nrows = num_rows)\n",
    "    bb, bb_cat = one_hot_encoder(bb, nan_as_category)\n",
    "    bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category)\n",
    "    \n",
    "    # Bureau balance: Perform aggregations and merge with bureau.csv\n",
    "    bb_aggregations = {'MONTHS_BALANCE': ['min', 'max', 'size']}\n",
    "    for col in bb_cat:\n",
    "        bb_aggregations[col] = ['mean']\n",
    "    bb_agg = bb.groupby('SK_ID_BUREAU').agg(bb_aggregations)\n",
    "    bb_agg.columns = pd.Index([e[0] + \"_\" + e[1].upper() for e in bb_agg.columns.tolist()])\n",
    "    bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')\n",
    "    bureau.drop(['SK_ID_BUREAU'], axis=1, inplace= True)\n",
    "    del bb, bb_agg\n",
    "    gc.collect()\n",
    "    \n",
    "    # Bureau and bureau_balance numeric features\n",
    "    num_aggregations = {\n",
    "        'DAYS_CREDIT': ['min', 'max', 'mean', 'var'],\n",
    "        'DAYS_CREDIT_ENDDATE': ['min', 'max', 'mean'],\n",
    "        'DAYS_CREDIT_UPDATE': ['mean'],\n",
    "        'CREDIT_DAY_OVERDUE': ['max', 'mean'],\n",
    "        'AMT_CREDIT_MAX_OVERDUE': ['mean'],\n",
    "        'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],\n",
    "        'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],\n",
    "        'AMT_CREDIT_SUM_OVERDUE': ['mean'],\n",
    "        'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],\n",
    "        'AMT_ANNUITY': ['max', 'mean'],\n",
    "        'CNT_CREDIT_PROLONG': ['sum'],\n",
    "        'MONTHS_BALANCE_MIN': ['min'],\n",
    "        'MONTHS_BALANCE_MAX': ['max'],\n",
    "        'MONTHS_BALANCE_SIZE': ['mean', 'sum']\n",
    "    }\n",
    "    # Bureau and bureau_balance categorical features\n",
    "    cat_aggregations = {}\n",
    "    for cat in bureau_cat: cat_aggregations[cat] = ['mean']\n",
    "    for cat in bb_cat: cat_aggregations[cat + \"_MEAN\"] = ['mean']\n",
    "    \n",
    "    bureau_agg = bureau.groupby('SK_ID_CURR').agg({**num_aggregations, **cat_aggregations})\n",
    "    bureau_agg.columns = pd.Index(['BURO_' + e[0] + \"_\" + e[1].upper() for e in bureau_agg.columns.tolist()])\n",
    "    # Bureau: Active credits - using only numerical aggregations\n",
    "    active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]\n",
    "    active_agg = active.groupby('SK_ID_CURR').agg(num_aggregations)\n",
    "    active_agg.columns = pd.Index(['ACTIVE_' + e[0] + \"_\" + e[1].upper() for e in active_agg.columns.tolist()])\n",
    "    bureau_agg = bureau_agg.join(active_agg, how='left', on='SK_ID_CURR')\n",
    "    del active, active_agg\n",
    "    gc.collect()\n",
    "    # Bureau: Closed credits - using only numerical aggregations\n",
    "    closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]\n",
    "    closed_agg = closed.groupby('SK_ID_CURR').agg(num_aggregations)\n",
    "    closed_agg.columns = pd.Index(['CLOSED_' + e[0] + \"_\" + e[1].upper() for e in closed_agg.columns.tolist()])\n",
    "    bureau_agg = bureau_agg.join(closed_agg, how='left', on='SK_ID_CURR')\n",
    "    del closed, closed_agg, bureau\n",
    "    gc.collect()\n",
    "    return bureau_agg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocess previous_applications.csv\n",
    "def previous_applications(num_rows = None, nan_as_category = True):\n",
    "    prev = pd.read_csv('../data/previous_application.csv', nrows = num_rows)\n",
    "    prev, cat_cols = one_hot_encoder(prev, nan_as_category= True)\n",
    "    # Days 365.243 values -> nan\n",
    "    prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)\n",
    "    prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)\n",
    "    prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)\n",
    "    prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)\n",
    "    prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)\n",
    "    # Add feature: value ask / value received percentage\n",
    "    prev['APP_CREDIT_PERC'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']\n",
    "    # Previous applications numeric features\n",
    "    num_aggregations = {\n",
    "        'AMT_ANNUITY': ['min', 'max', 'mean'],\n",
    "        'AMT_APPLICATION': ['min', 'max', 'mean'],\n",
    "        'AMT_CREDIT': ['min', 'max', 'mean'],\n",
    "        'APP_CREDIT_PERC': ['min', 'max', 'mean', 'var'],\n",
    "        'AMT_DOWN_PAYMENT': ['min', 'max', 'mean'],\n",
    "        'AMT_GOODS_PRICE': ['min', 'max', 'mean'],\n",
    "        'HOUR_APPR_PROCESS_START': ['min', 'max', 'mean'],\n",
    "        'RATE_DOWN_PAYMENT': ['min', 'max', 'mean'],\n",
    "        'DAYS_DECISION': ['min', 'max', 'mean'],\n",
    "        'CNT_PAYMENT': ['mean', 'sum'],\n",
    "    }\n",
    "    # Previous applications categorical features\n",
    "    cat_aggregations = {}\n",
    "    for cat in cat_cols:\n",
    "        cat_aggregations[cat] = ['mean']\n",
    "    \n",
    "    prev_agg = prev.groupby('SK_ID_CURR').agg({**num_aggregations, **cat_aggregations})\n",
    "    prev_agg.columns = pd.Index(['PREV_' + e[0] + \"_\" + e[1].upper() for e in prev_agg.columns.tolist()])\n",
    "    # Previous Applications: Approved Applications - only numerical features\n",
    "    approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]\n",
    "    approved_agg = approved.groupby('SK_ID_CURR').agg(num_aggregations)\n",
    "    approved_agg.columns = pd.Index(['APPROVED_' + e[0] + \"_\" + e[1].upper() for e in approved_agg.columns.tolist()])\n",
    "    prev_agg = prev_agg.join(approved_agg, how='left', on='SK_ID_CURR')\n",
    "    # Previous Applications: Refused Applications - only numerical features\n",
    "    refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]\n",
    "    refused_agg = refused.groupby('SK_ID_CURR').agg(num_aggregations)\n",
    "    refused_agg.columns = pd.Index(['REFUSED_' + e[0] + \"_\" + e[1].upper() for e in refused_agg.columns.tolist()])\n",
    "    prev_agg = prev_agg.join(refused_agg, how='left', on='SK_ID_CURR')\n",
    "    del refused, refused_agg, approved, approved_agg, prev\n",
    "    gc.collect()\n",
    "    return prev_agg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocess POS_CASH_balance.csv\n",
    "def pos_cash(num_rows = None, nan_as_category = True):\n",
    "    pos = pd.read_csv('../data/POS_CASH_balance.csv', nrows = num_rows)\n",
    "    pos, cat_cols = one_hot_encoder(pos, nan_as_category= True)\n",
    "    # Features\n",
    "    aggregations = {\n",
    "        'MONTHS_BALANCE': ['max', 'mean', 'size'],\n",
    "        'SK_DPD': ['max', 'mean'],\n",
    "        'SK_DPD_DEF': ['max', 'mean']\n",
    "    }\n",
    "    for cat in cat_cols:\n",
    "        aggregations[cat] = ['mean']\n",
    "    \n",
    "    pos_agg = pos.groupby('SK_ID_CURR').agg(aggregations)\n",
    "    pos_agg.columns = pd.Index(['POS_' + e[0] + \"_\" + e[1].upper() for e in pos_agg.columns.tolist()])\n",
    "    # Count pos cash accounts\n",
    "    pos_agg['POS_COUNT'] = pos.groupby('SK_ID_CURR').size()\n",
    "    del pos\n",
    "    gc.collect()\n",
    "    return pos_agg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocess installments_payments.csv\n",
    "def installments_payments(num_rows = None, nan_as_category = True):\n",
    "    ins = pd.read_csv('../data/installments_payments.csv', nrows = num_rows)\n",
    "    ins, cat_cols = one_hot_encoder(ins, nan_as_category= True)\n",
    "    # Percentage and difference paid in each installment (amount paid and installment value)\n",
    "    ins['PAYMENT_PERC'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']\n",
    "    ins['PAYMENT_DIFF'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT']\n",
    "    # Days past due and days before due (no negative values)\n",
    "    ins['DPD'] = ins['DAYS_ENTRY_PAYMENT'] - ins['DAYS_INSTALMENT']\n",
    "    ins['DBD'] = ins['DAYS_INSTALMENT'] - ins['DAYS_ENTRY_PAYMENT']\n",
    "    ins['DPD'] = ins['DPD'].apply(lambda x: x if x > 0 else 0)\n",
    "    ins['DBD'] = ins['DBD'].apply(lambda x: x if x > 0 else 0)\n",
    "    # Features: Perform aggregations\n",
    "    aggregations = {\n",
    "        'NUM_INSTALMENT_VERSION': ['nunique'],\n",
    "        'DPD': ['max', 'mean', 'sum'],\n",
    "        'DBD': ['max', 'mean', 'sum'],\n",
    "        'PAYMENT_PERC': ['max', 'mean', 'sum', 'var'],\n",
    "        'PAYMENT_DIFF': ['max', 'mean', 'sum', 'var'],\n",
    "        'AMT_INSTALMENT': ['max', 'mean', 'sum'],\n",
    "        'AMT_PAYMENT': ['min', 'max', 'mean', 'sum'],\n",
    "        'DAYS_ENTRY_PAYMENT': ['max', 'mean', 'sum']\n",
    "    }\n",
    "    for cat in cat_cols:\n",
    "        aggregations[cat] = ['mean']\n",
    "    ins_agg = ins.groupby('SK_ID_CURR').agg(aggregations)\n",
    "    ins_agg.columns = pd.Index(['INSTAL_' + e[0] + \"_\" + e[1].upper() for e in ins_agg.columns.tolist()])\n",
    "    # Count installments accounts\n",
    "    ins_agg['INSTAL_COUNT'] = ins.groupby('SK_ID_CURR').size()\n",
    "    del ins\n",
    "    gc.collect()\n",
    "    return ins_agg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocess credit_card_balance.csv\n",
    "def credit_card_balance(num_rows = None, nan_as_category = True):\n",
    "    cc = pd.read_csv('../data/credit_card_balance.csv', nrows = num_rows)\n",
    "    cc, cat_cols = one_hot_encoder(cc, nan_as_category= True)\n",
    "    # General aggregations\n",
    "    cc.drop(['SK_ID_PREV'], axis= 1, inplace = True)\n",
    "    cc_agg = cc.groupby('SK_ID_CURR').agg(['min', 'max', 'mean', 'sum', 'var'])\n",
    "    cc_agg.columns = pd.Index(['CC_' + e[0] + \"_\" + e[1].upper() for e in cc_agg.columns.tolist()])\n",
    "    # Count credit card lines\n",
    "    cc_agg['CC_COUNT'] = cc.groupby('SK_ID_CURR').size()\n",
    "    del cc\n",
    "    gc.collect()\n",
    "    return cc_agg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fill_mark_na(df):\n",
    "    \"\"\"Fill NAs with the median by column, adding an additional indicator column to mark\n",
    "    where they occur\"\"\"\n",
    "    #not_target = [c for c in df.columns.values if c != 'TARGET']\n",
    "    not_target_data = df.drop(labels='TARGET', axis=1)\n",
    "    nulls = not_target_data.apply(pd.isnull, axis=1)\n",
    "    nulls = nulls.astype(int)\n",
    "    \n",
    "    not_target_data.replace(np.inf, np.nan, inplace=True)\n",
    "    not_target_data.replace(-np.inf, np.nan, inplace=True)\n",
    "    not_target_data.fillna(not_target_data.median(), inplace=True)\n",
    "    df[not_target_data.columns.values] = not_target_data.values\n",
    "    \n",
    "    data_types = df.dtypes\n",
    "    sel = data_types == np.float64\n",
    "    df[data_types.index[sel]] = df.astype(np.float32)\n",
    "    \n",
    "    check_nulls = not_target_data.isnull().any()\n",
    "    check_nulls = check_nulls[check_nulls == True].index.values\n",
    "    \n",
    "    df.drop(labels=check_nulls, axis=1, inplace=True)\n",
    "    nulls.drop(labels=check_nulls, axis=1, inplace=True)\n",
    "    \n",
    "    nulls.columns = [c+'_null_ind' for c in nulls.columns.values]\n",
    "    #df = pd.concat([df, nulls])\n",
    "    \n",
    "    print(df.shape)\n",
    "    \n",
    "    del not_target_data\n",
    "    gc.collect()\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# parameter space for Bayesian optimization:\n",
    "space  = [Integer(3, 10, name='max_depth'),\n",
    "          Integer(6, 80, name='num_leaves'),\n",
    "          Integer(50, 200, name='min_child_samples'),\n",
    "          Integer(75, 125, name='max_bin'),\n",
    "          Real(1, 400,  name='scale_pos_weight'),\n",
    "          Real(0.6, 0.8, name='subsample'),\n",
    "          Real(0.8, 0.97, name='colsample_bytree')\n",
    "         ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "# objective function to minimize for this problem:\n",
    "def objective(space):\n",
    "    \n",
    "    params = {'max_depth': space[0], \n",
    "          'num_leaves': space[1], \n",
    "          'min_child_samples': space[2],\n",
    "          'max_bin': space[3],\n",
    "          'scale_pos_weight': space[4],\n",
    "            'subsample': space[5],\n",
    "            'colsample_bytree': space[6],\n",
    "            'n_estimators': 1000,\n",
    "            'metric':'auc',\n",
    "            'nthread': 8,\n",
    "            'boosting_type': 'dart',\n",
    "            'objective': 'binary',\n",
    "            'learning_rate':0.02,\n",
    "            'min_child_weight': 0,\n",
    "            'min_split_gain': 0,\n",
    "            'subsample_freq': 1}\n",
    "    \n",
    "    early_stopping_rounds = 50\n",
    "    num_boost_round       = 1000\n",
    "    \n",
    "    #X_train.columns = [str(i) for i in range(len(X_train.columns.values))]\n",
    "    feature_set = X_train.columns.values.tolist()\n",
    "    #print(feature_set == X_train.columns.values.tolist())\n",
    "    \n",
    "    #cats = X_train.dtypes == 'object'\n",
    "    #categorical = X_train[cats.index[cats]].columns.values\n",
    "    #categorical = categorical.tolist()\n",
    "    #feature_set = [c.replace(' ', '_') for c in feature_set]\n",
    "    categorical = ['FLAG_DOCUMENT_7']\n",
    "    X_train.columns = feature_set\n",
    "\n",
    "    # Fit model on feature_set and calculate validation AUROC\n",
    "    xgtrain = lgb.Dataset(X_train[feature_set].values, label=y_train, \n",
    "                          feature_name=feature_set)\n",
    "    \n",
    "    xgvalid = lgb.Dataset(X_test[feature_set].values, label=y_test, \n",
    "                          feature_name=feature_set)\n",
    "    \n",
    "    evals_results = {}\n",
    "    model_lgb     = lgb.train(params, xgtrain, valid_sets=[xgtrain, xgvalid], \n",
    "                              valid_names=['train','valid'], \n",
    "                               evals_result=evals_results, \n",
    "                               num_boost_round=num_boost_round,\n",
    "                                early_stopping_rounds=early_stopping_rounds,\n",
    "                               verbose_eval=None, feval=None)\n",
    "    \n",
    "    auc = roc_auc_score(y_test, model_lgb.predict(X_test[model_lgb.feature_name()]))\n",
    "    \n",
    "    print('\\nAUROC.....',-auc,\".....iter.....\", model_lgb.current_iteration())\n",
    "    \n",
    "    gc.collect()\n",
    "    \n",
    "    return -auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LightGBM GBDT with KFold or Stratified KFold\n",
    "# Parameters from Tilii kernel: https://www.kaggle.com/tilii7/olivier-lightgbm-parameters-by-bayesian-opt/code\n",
    "def kfold_lightgbm(df, num_folds, stratified = False, debug= False):\n",
    "    # Divide in training/validation and test data\n",
    "    train_df = df[df['TARGET'].notnull()]\n",
    "    test_df = df[df['TARGET'].isnull()]\n",
    "    print(\"Starting LightGBM. Train shape: {}, test shape: {}\".format(train_df.shape, test_df.shape))\n",
    "    del df\n",
    "    gc.collect()\n",
    "    # Cross validation model\n",
    "    if stratified:\n",
    "        folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=1001)\n",
    "    else:\n",
    "        folds = KFold(n_splits= num_folds, shuffle=True, random_state=1001)\n",
    "    # Create arrays and dataframes to store results\n",
    "    oof_preds = np.zeros(train_df.shape[0])\n",
    "    sub_preds = np.zeros(test_df.shape[0])\n",
    "    feature_importance_df = pd.DataFrame()\n",
    "    feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]\n",
    "    \n",
    "    for n_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df[feats], train_df['TARGET'])):\n",
    "        train_x, train_y = train_df[feats].iloc[train_idx], train_df['TARGET'].iloc[train_idx]\n",
    "        valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['TARGET'].iloc[valid_idx]\n",
    "\n",
    "        # LightGBM parameters found by Bayesian optimization\n",
    "        clf = LGBMClassifier(\n",
    "            nthread=4,\n",
    "            n_estimators=10000,\n",
    "            learning_rate=0.02,\n",
    "            num_leaves=50,\n",
    "            colsample_bytree=0.9497036,\n",
    "            subsample=0.8715623,\n",
    "            max_depth=5,\n",
    "            reg_alpha=0.041545473,\n",
    "            reg_lambda=0.0735294,\n",
    "            min_split_gain=0.0222415,\n",
    "            min_child_weight=39.3259775,\n",
    "            silent=-1,\n",
    "            verbose=-1, )\n",
    "\n",
    "        clf.fit(train_x, train_y, eval_set=[(train_x, train_y), (valid_x, valid_y)], \n",
    "            eval_metric= 'auc', verbose= 100, early_stopping_rounds= 200)\n",
    "\n",
    "        oof_preds[valid_idx] = clf.predict_proba(valid_x, num_iteration=clf.best_iteration_)[:, 1]\n",
    "        sub_preds += clf.predict_proba(test_df[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits\n",
    "\n",
    "        fold_importance_df = pd.DataFrame()\n",
    "        fold_importance_df[\"feature\"] = feats\n",
    "        fold_importance_df[\"importance\"] = clf.feature_importances_\n",
    "        fold_importance_df[\"fold\"] = n_fold + 1\n",
    "        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n",
    "        print('Fold %2d AUC : %.6f' % (n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))\n",
    "        del clf, train_x, train_y, valid_x, valid_y\n",
    "        gc.collect()\n",
    "\n",
    "    print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))\n",
    "    # Write submission file and plot feature importance\n",
    "    if not debug:\n",
    "        test_df['TARGET'] = sub_preds\n",
    "        test_df[['SK_ID_CURR', 'TARGET']].to_csv(submission_file_name, index= False)\n",
    "    display_importances(feature_importance_df)\n",
    "    return feature_importance_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def kfold_randomforest(df, num_folds, stratified = False, debug= False):\n",
    "    # Divide in training/validation and test data\n",
    "    train_df = df[df['TARGET'].notnull()]\n",
    "    test_df = df[df['TARGET'].isnull()]\n",
    "    print(\"Starting Random Forest. Train shape: {}, test shape: {}\".format(train_df.shape, test_df.shape))\n",
    "    del df\n",
    "    gc.collect()\n",
    "    # Cross validation model\n",
    "    if stratified:\n",
    "        folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=1001)\n",
    "    else:\n",
    "        folds = KFold(n_splits= num_folds, shuffle=True, random_state=1001)\n",
    "    # Create arrays and dataframes to store results\n",
    "    oof_preds = np.zeros(train_df.shape[0])\n",
    "    sub_preds = np.zeros(test_df.shape[0])\n",
    "    feature_importance_df = pd.DataFrame()\n",
    "    feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]\n",
    "    \n",
    "    for n_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df[feats], train_df['TARGET'])):\n",
    "        train_x, train_y = train_df[feats].iloc[train_idx], train_df['TARGET'].iloc[train_idx]\n",
    "        valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['TARGET'].iloc[valid_idx]\n",
    "\n",
    "        # LightGBM parameters found by Bayesian optimization\n",
    "        clf = RandomForestClassifier(\n",
    "            n_estimators=50,\n",
    "            max_depth=10,\n",
    "            class_weight='balanced',\n",
    "            n_jobs=4)\n",
    "        \n",
    "        clf.fit(train_x, train_y)\n",
    "\n",
    "        #clf.fit(train_x, train_y, eval_set=[(train_x, train_y), (valid_x, valid_y)], \n",
    "        #    eval_metric= 'auc', verbose= 100, early_stopping_rounds= 200)\n",
    "\n",
    "        oof_preds[valid_idx] = clf.predict_proba(valid_x)[:, 1]\n",
    "        sub_preds += clf.predict_proba(test_df[feats])[:, 1] / folds.n_splits\n",
    "\n",
    "        fold_importance_df = pd.DataFrame()\n",
    "        fold_importance_df[\"feature\"] = feats\n",
    "        fold_importance_df[\"importance\"] = clf.feature_importances_\n",
    "        fold_importance_df[\"fold\"] = n_fold + 1\n",
    "        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n",
    "        print('Fold %2d AUC : %.6f' % (n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))\n",
    "        del clf, train_x, train_y, valid_x, valid_y\n",
    "        gc.collect()\n",
    "\n",
    "    print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))\n",
    "    # Write submission file and plot feature importance\n",
    "    if not debug:\n",
    "        test_df['TARGET'] = sub_preds\n",
    "        test_df[['SK_ID_CURR', 'TARGET']].to_csv(submission_file_name, index= False)\n",
    "    display_importances(feature_importance_df)\n",
    "    return feature_importance_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display/plot feature importance\n",
    "def display_importances(feature_importance_df_):\n",
    "    cols = feature_importance_df_[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(by=\"importance\", ascending=False)[:40].index\n",
    "    best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]\n",
    "    plt.figure(figsize=(8, 10))\n",
    "    sns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False))\n",
    "    plt.title('LightGBM Features (avg over folds)')\n",
    "    plt.tight_layout()\n",
    "    plt.savefig('lgbm_importances01.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(debug = False, optimize_params = False, model='lightgbm', from_file=True):\n",
    "    num_rows = 10000 if debug else None\n",
    "    \n",
    "    if from_file:\n",
    "        df = pd.read_csv('../data/tr_te_processed.csv', nrows=num_rows)\n",
    "        df.drop(labels=['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3', \n",
    "                        'EXT_SOURCE_1_REGION_CITY', 'EXT_SOURCE_2_REGION_CITY',\n",
    "                        'EXT_SOURCE_3_REGION_CITY'], axis=1, inplace = True)\n",
    "    \n",
    "    else:\n",
    "        df = application_train_test(num_rows)\n",
    "        with timer(\"Process bureau and bureau_balance\"):\n",
    "            bureau = bureau_and_balance(num_rows)\n",
    "            print(\"Bureau df shape:\", bureau.shape)\n",
    "            df = df.join(bureau, how='left', on='SK_ID_CURR')\n",
    "            del bureau\n",
    "            gc.collect()\n",
    "        with timer(\"Process previous_applications\"):\n",
    "            prev = previous_applications(num_rows)\n",
    "            print(\"Previous applications df shape:\", prev.shape)\n",
    "            df = df.join(prev, how='left', on='SK_ID_CURR')\n",
    "            del prev\n",
    "            gc.collect()\n",
    "        with timer(\"Process POS-CASH balance\"):\n",
    "            pos = pos_cash(num_rows)\n",
    "            print(\"Pos-cash balance df shape:\", pos.shape)\n",
    "            df = df.join(pos, how='left', on='SK_ID_CURR')\n",
    "            del pos\n",
    "            gc.collect()\n",
    "        with timer(\"Process installments payments\"):\n",
    "            ins = installments_payments(num_rows)\n",
    "            print(\"Installments payments df shape:\", ins.shape)\n",
    "            df = df.join(ins, how='left', on='SK_ID_CURR')\n",
    "            del ins\n",
    "            gc.collect()\n",
    "        with timer(\"Process credit card balance\"):\n",
    "            cc = credit_card_balance(num_rows)\n",
    "            print(\"Credit card balance df shape:\", cc.shape)\n",
    "            df = df.join(cc, how='left', on='SK_ID_CURR')\n",
    "            del cc\n",
    "            gc.collect()\n",
    "        with timer(\"Replace missing values\"):\n",
    "            df = fill_mark_na(df)\n",
    "            print(\"Final dataframe df shape:\", df.shape)\n",
    "            gc.collect()\n",
    "            \n",
    "    if optimize_params:\n",
    "        return df\n",
    "    \n",
    "    else:\n",
    "        with timer(\"Run Model with kfold\"):\n",
    "            if not from_file:\n",
    "                df.to_csv('../data/tr_te_processed.csv')\n",
    "            \n",
    "            if model == 'lightgbm':\n",
    "                feat_importance = kfold_lightgbm(df, num_folds= 5, stratified= False, debug= debug) \n",
    "            elif model == 'rf':\n",
    "                feat_importance = kfold_randomforest(df, num_folds= 5, stratified= False, debug= debug)\n",
    "            \n",
    "            return feat_importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train samples: 10000, test samples: 10000\n",
      "Bureau df shape: (2011, 108)\n",
      "Process bureau and bureau_balance - done in 0s\n",
      "Previous applications df shape: (9734, 242)\n",
      "Process previous_applications - done in 1s\n",
      "Pos-cash balance df shape: (9494, 15)\n",
      "Process POS-CASH balance - done in 0s\n",
      "Installments payments df shape: (8893, 26)\n",
      "Process installments payments - done in 0s\n",
      "Credit card balance df shape: (9520, 131)\n",
      "Process credit card balance - done in 0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mattwinkler/anaconda3/envs/credit-analysis/lib/python3.6/site-packages/numpy/lib/nanfunctions.py:907: RuntimeWarning: All-NaN slice encountered\n",
      "  result = np.apply_along_axis(_nanmedian1d, axis, a, overwrite_input)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20000, 811)\n",
      "Final dataframe df shape: (20000, 811)\n",
      "Replace missing values - done in 21s\n",
      "Full model run - done in 24s\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    submission_file_name = \"submission_kernel02.csv\"\n",
    "    with timer(\"Full model run\"):\n",
    "        result = main(debug=True, optimize_params=True, \n",
    "                               model='lightgbm', from_file=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = result\n",
    "train_df = df[df['TARGET'].notnull()]\n",
    "y = train_df['TARGET']\n",
    "feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index', 'index_null_ind']]\n",
    "train_df = train_df[feats].copy()\n",
    "    \n",
    "X_train, X_test, y_train, y_test = train_test_split(\\\n",
    "            train_df, y, test_size=0.33, random_state=42)\n",
    "\n",
    "X_train.columns = [c.replace(' ', '_') for c in X_train.columns.values.tolist()]\n",
    "\n",
    "#'FONDKAPREMONT_MODE_not_specified' in X_train.columns.values.tolist()\n",
    "#sorted(X_train.columns.values.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "\"['FONDKAPREMONT_MODE_not_specified' 'FONDKAPREMONT_MODE_org_spec_account'\\n 'FONDKAPREMONT_MODE_reg_oper_account'\\n 'FONDKAPREMONT_MODE_reg_oper_spec_account'\\n 'HOUSETYPE_MODE_block_of_flats' 'HOUSETYPE_MODE_specific_housing'\\n 'HOUSETYPE_MODE_terraced_house' 'NAME_CONTRACT_TYPE_Cash_loans'\\n 'NAME_CONTRACT_TYPE_Revolving_loans'\\n 'NAME_EDUCATION_TYPE_Academic_degree'\\n 'NAME_EDUCATION_TYPE_Higher_education'\\n 'NAME_EDUCATION_TYPE_Incomplete_higher'\\n 'NAME_EDUCATION_TYPE_Lower_secondary'\\n 'NAME_EDUCATION_TYPE_Secondary_/_secondary_special'\\n 'NAME_FAMILY_STATUS_Civil_marriage'\\n 'NAME_FAMILY_STATUS_Single_/_not_married'\\n 'NAME_HOUSING_TYPE_Co-op_apartment' 'NAME_HOUSING_TYPE_House_/_apartment'\\n 'NAME_HOUSING_TYPE_Municipal_apartment'\\n 'NAME_HOUSING_TYPE_Office_apartment' 'NAME_HOUSING_TYPE_Rented_apartment'\\n 'NAME_HOUSING_TYPE_With_parents' 'NAME_INCOME_TYPE_Commercial_associate'\\n 'NAME_INCOME_TYPE_State_servant' 'NAME_TYPE_SUITE_Group_of_people'\\n 'NAME_TYPE_SUITE_Spouse,_partner' 'OCCUPATION_TYPE_Cleaning_staff'\\n 'OCCUPATION_TYPE_Cooking_staff' 'OCCUPATION_TYPE_Core_staff'\\n 'OCCUPATION_TYPE_HR_staff' 'OCCUPATION_TYPE_High_skill_tech_staff'\\n 'OCCUPATION_TYPE_IT_staff' 'OCCUPATION_TYPE_Low-skill_Laborers'\\n 'OCCUPATION_TYPE_Medicine_staff' 'OCCUPATION_TYPE_Private_service_staff'\\n 'OCCUPATION_TYPE_Realty_agents' 'OCCUPATION_TYPE_Sales_staff'\\n 'OCCUPATION_TYPE_Security_staff' 'OCCUPATION_TYPE_Waiters/barmen_staff'\\n 'ORGANIZATION_TYPE_Business_Entity_Type_1'\\n 'ORGANIZATION_TYPE_Business_Entity_Type_2'\\n 'ORGANIZATION_TYPE_Business_Entity_Type_3'\\n 'ORGANIZATION_TYPE_Industry:_type_1'\\n 'ORGANIZATION_TYPE_Industry:_type_10'\\n 'ORGANIZATION_TYPE_Industry:_type_11'\\n 'ORGANIZATION_TYPE_Industry:_type_12'\\n 'ORGANIZATION_TYPE_Industry:_type_13'\\n 'ORGANIZATION_TYPE_Industry:_type_2' 'ORGANIZATION_TYPE_Industry:_type_3'\\n 'ORGANIZATION_TYPE_Industry:_type_4' 'ORGANIZATION_TYPE_Industry:_type_5'\\n 'ORGANIZATION_TYPE_Industry:_type_6' 'ORGANIZATION_TYPE_Industry:_type_7'\\n 'ORGANIZATION_TYPE_Industry:_type_8' 'ORGANIZATION_TYPE_Industry:_type_9'\\n 'ORGANIZATION_TYPE_Legal_Services'\\n 'ORGANIZATION_TYPE_Security_Ministries' 'ORGANIZATION_TYPE_Trade:_type_1'\\n 'ORGANIZATION_TYPE_Trade:_type_2' 'ORGANIZATION_TYPE_Trade:_type_3'\\n 'ORGANIZATION_TYPE_Trade:_type_4' 'ORGANIZATION_TYPE_Trade:_type_5'\\n 'ORGANIZATION_TYPE_Trade:_type_6' 'ORGANIZATION_TYPE_Trade:_type_7'\\n 'ORGANIZATION_TYPE_Transport:_type_1'\\n 'ORGANIZATION_TYPE_Transport:_type_2'\\n 'ORGANIZATION_TYPE_Transport:_type_3'\\n 'ORGANIZATION_TYPE_Transport:_type_4' 'WALLSMATERIAL_MODE_Stone,_brick'\\n 'BURO_CREDIT_ACTIVE_Bad_debt_MEAN' 'BURO_CREDIT_CURRENCY_currency_1_MEAN'\\n 'BURO_CREDIT_CURRENCY_currency_2_MEAN'\\n 'BURO_CREDIT_CURRENCY_currency_4_MEAN'\\n 'BURO_CREDIT_TYPE_Another_type_of_loan_MEAN'\\n 'BURO_CREDIT_TYPE_Car_loan_MEAN' 'BURO_CREDIT_TYPE_Consumer_credit_MEAN'\\n 'BURO_CREDIT_TYPE_Credit_card_MEAN'\\n 'BURO_CREDIT_TYPE_Loan_for_business_development_MEAN'\\n 'BURO_CREDIT_TYPE_Loan_for_working_capital_replenishment_MEAN'\\n 'BURO_CREDIT_TYPE_Real_estate_loan_MEAN'\\n 'BURO_CREDIT_TYPE_Unknown_type_of_loan_MEAN'\\n 'PREV_NAME_CONTRACT_TYPE_Cash_loans_MEAN'\\n 'PREV_NAME_CONTRACT_TYPE_Consumer_loans_MEAN'\\n 'PREV_NAME_CONTRACT_TYPE_Revolving_loans_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Building_a_house_or_an_annex_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Business_development_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Buying_a_garage_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Buying_a_holiday_home_/_land_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Buying_a_home_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Buying_a_new_car_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Buying_a_used_car_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Car_repairs_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Everyday_expenses_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Gasification_/_water_supply_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Payments_on_other_loans_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Purchase_of_electronic_equipment_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Urgent_needs_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Wedding_/_gift_/_holiday_MEAN'\\n 'PREV_NAME_CONTRACT_STATUS_Unused_offer_MEAN'\\n 'PREV_NAME_PAYMENT_TYPE_Cash_through_the_bank_MEAN'\\n 'PREV_NAME_PAYMENT_TYPE_Cashless_from_the_account_of_the_employer_MEAN'\\n 'PREV_NAME_PAYMENT_TYPE_Non-cash_from_your_account_MEAN'\\n 'PREV_NAME_TYPE_SUITE_Group_of_people_MEAN'\\n 'PREV_NAME_TYPE_SUITE_Spouse,_partner_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Auto_Accessories_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Clothing_and_Accessories_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Construction_Materials_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Consumer_Electronics_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Direct_Sales_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Medical_Supplies_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Office_Appliances_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Photo_/_Cinema_Equipment_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Sport_and_Leisure_MEAN'\\n 'PREV_CHANNEL_TYPE_AP+_(Cash_loan)_MEAN'\\n 'PREV_CHANNEL_TYPE_Car_dealer_MEAN'\\n 'PREV_CHANNEL_TYPE_Channel_of_corporate_sales_MEAN'\\n 'PREV_CHANNEL_TYPE_Contact_center_MEAN'\\n 'PREV_CHANNEL_TYPE_Credit_and_cash_offices_MEAN'\\n 'PREV_CHANNEL_TYPE_Regional_/_Local_MEAN'\\n 'PREV_NAME_SELLER_INDUSTRY_Auto_technology_MEAN'\\n 'PREV_NAME_SELLER_INDUSTRY_Consumer_electronics_MEAN'\\n 'PREV_NAME_SELLER_INDUSTRY_MLM_partners_MEAN'\\n 'PREV_PRODUCT_COMBINATION_Card_Street_MEAN'\\n 'PREV_PRODUCT_COMBINATION_Card_X-Sell_MEAN'\\n 'PREV_PRODUCT_COMBINATION_Cash_Street:_high_MEAN'\\n 'PREV_PRODUCT_COMBINATION_Cash_Street:_low_MEAN'\\n 'PREV_PRODUCT_COMBINATION_Cash_Street:_middle_MEAN'\\n 'PREV_PRODUCT_COMBINATION_Cash_X-Sell:_high_MEAN'\\n 'PREV_PRODUCT_COMBINATION_Cash_X-Sell:_low_MEAN'\\n 'PREV_PRODUCT_COMBINATION_Cash_X-Sell:_middle_MEAN'\\n 'PREV_PRODUCT_COMBINATION_POS_household_with_interest_MEAN'\\n 'PREV_PRODUCT_COMBINATION_POS_household_without_interest_MEAN'\\n 'PREV_PRODUCT_COMBINATION_POS_industry_with_interest_MEAN'\\n 'PREV_PRODUCT_COMBINATION_POS_industry_without_interest_MEAN'\\n 'PREV_PRODUCT_COMBINATION_POS_mobile_with_interest_MEAN'\\n 'PREV_PRODUCT_COMBINATION_POS_mobile_without_interest_MEAN'\\n 'PREV_PRODUCT_COMBINATION_POS_other_with_interest_MEAN'\\n 'PREV_PRODUCT_COMBINATION_POS_others_without_interest_MEAN'\\n 'POS_NAME_CONTRACT_STATUS_Returned_to_the_store_MEAN'\\n 'CC_NAME_CONTRACT_STATUS_Sent_proposal_MIN'\\n 'CC_NAME_CONTRACT_STATUS_Sent_proposal_MAX'\\n 'CC_NAME_CONTRACT_STATUS_Sent_proposal_MEAN'\\n 'CC_NAME_CONTRACT_STATUS_Sent_proposal_SUM'\\n 'CC_NAME_CONTRACT_STATUS_Sent_proposal_VAR'] not in index\"",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-218-a778343a49c6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m res_gp = gp_minimize(objective, space, n_calls=20,\n\u001b[0;32m----> 2\u001b[0;31m                      random_state=0, n_random_starts=10)\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\"Best score=%.4f\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mres_gp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfun\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/credit-analysis/lib/python3.6/site-packages/skopt/optimizer/gp.py\u001b[0m in \u001b[0;36mgp_minimize\u001b[0;34m(func, dimensions, base_estimator, n_calls, n_random_starts, acq_func, acq_optimizer, x0, y0, random_state, verbose, callback, n_points, n_restarts_optimizer, xi, kappa, noise, n_jobs)\u001b[0m\n\u001b[1;32m    226\u001b[0m         \u001b[0mn_restarts_optimizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_restarts_optimizer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    227\u001b[0m         \u001b[0mx0\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mx0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my0\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrng\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 228\u001b[0;31m         callback=callback, n_jobs=n_jobs)\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/credit-analysis/lib/python3.6/site-packages/skopt/optimizer/base.py\u001b[0m in \u001b[0;36mbase_minimize\u001b[0;34m(func, dimensions, base_estimator, n_calls, n_random_starts, acq_func, acq_optimizer, x0, y0, random_state, verbose, callback, n_points, n_restarts_optimizer, xi, kappa, n_jobs)\u001b[0m\n\u001b[1;32m    246\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_calls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m         \u001b[0mnext_x\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 248\u001b[0;31m         \u001b[0mnext_y\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnext_x\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    249\u001b[0m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnext_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnext_y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    250\u001b[0m         \u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspecs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspecs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-213-f87541a76895>\u001b[0m in \u001b[0;36mobjective\u001b[0;34m(space)\u001b[0m\n\u001b[1;32m     38\u001b[0m                           feature_name=feature_set)\n\u001b[1;32m     39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m     xgvalid = lgb.Dataset(X_test[feature_set].values, label=y_test, \n\u001b[0m\u001b[1;32m     41\u001b[0m                           feature_name=feature_set)\n\u001b[1;32m     42\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/credit-analysis/lib/python3.6/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2680\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mSeries\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mIndex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2681\u001b[0m             \u001b[0;31m# either boolean or fancy integer index\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2682\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2683\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2684\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_frame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/credit-analysis/lib/python3.6/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m_getitem_array\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2724\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_take\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2725\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2726\u001b[0;31m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_convert_to_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2727\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_take\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2728\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/credit-analysis/lib/python3.6/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_convert_to_indexer\u001b[0;34m(self, obj, axis, is_setter)\u001b[0m\n\u001b[1;32m   1325\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1326\u001b[0m                     raise KeyError('{mask} not in index'\n\u001b[0;32m-> 1327\u001b[0;31m                                    .format(mask=objarr[mask]))\n\u001b[0m\u001b[1;32m   1328\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1329\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_values_from_object\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: \"['FONDKAPREMONT_MODE_not_specified' 'FONDKAPREMONT_MODE_org_spec_account'\\n 'FONDKAPREMONT_MODE_reg_oper_account'\\n 'FONDKAPREMONT_MODE_reg_oper_spec_account'\\n 'HOUSETYPE_MODE_block_of_flats' 'HOUSETYPE_MODE_specific_housing'\\n 'HOUSETYPE_MODE_terraced_house' 'NAME_CONTRACT_TYPE_Cash_loans'\\n 'NAME_CONTRACT_TYPE_Revolving_loans'\\n 'NAME_EDUCATION_TYPE_Academic_degree'\\n 'NAME_EDUCATION_TYPE_Higher_education'\\n 'NAME_EDUCATION_TYPE_Incomplete_higher'\\n 'NAME_EDUCATION_TYPE_Lower_secondary'\\n 'NAME_EDUCATION_TYPE_Secondary_/_secondary_special'\\n 'NAME_FAMILY_STATUS_Civil_marriage'\\n 'NAME_FAMILY_STATUS_Single_/_not_married'\\n 'NAME_HOUSING_TYPE_Co-op_apartment' 'NAME_HOUSING_TYPE_House_/_apartment'\\n 'NAME_HOUSING_TYPE_Municipal_apartment'\\n 'NAME_HOUSING_TYPE_Office_apartment' 'NAME_HOUSING_TYPE_Rented_apartment'\\n 'NAME_HOUSING_TYPE_With_parents' 'NAME_INCOME_TYPE_Commercial_associate'\\n 'NAME_INCOME_TYPE_State_servant' 'NAME_TYPE_SUITE_Group_of_people'\\n 'NAME_TYPE_SUITE_Spouse,_partner' 'OCCUPATION_TYPE_Cleaning_staff'\\n 'OCCUPATION_TYPE_Cooking_staff' 'OCCUPATION_TYPE_Core_staff'\\n 'OCCUPATION_TYPE_HR_staff' 'OCCUPATION_TYPE_High_skill_tech_staff'\\n 'OCCUPATION_TYPE_IT_staff' 'OCCUPATION_TYPE_Low-skill_Laborers'\\n 'OCCUPATION_TYPE_Medicine_staff' 'OCCUPATION_TYPE_Private_service_staff'\\n 'OCCUPATION_TYPE_Realty_agents' 'OCCUPATION_TYPE_Sales_staff'\\n 'OCCUPATION_TYPE_Security_staff' 'OCCUPATION_TYPE_Waiters/barmen_staff'\\n 'ORGANIZATION_TYPE_Business_Entity_Type_1'\\n 'ORGANIZATION_TYPE_Business_Entity_Type_2'\\n 'ORGANIZATION_TYPE_Business_Entity_Type_3'\\n 'ORGANIZATION_TYPE_Industry:_type_1'\\n 'ORGANIZATION_TYPE_Industry:_type_10'\\n 'ORGANIZATION_TYPE_Industry:_type_11'\\n 'ORGANIZATION_TYPE_Industry:_type_12'\\n 'ORGANIZATION_TYPE_Industry:_type_13'\\n 'ORGANIZATION_TYPE_Industry:_type_2' 'ORGANIZATION_TYPE_Industry:_type_3'\\n 'ORGANIZATION_TYPE_Industry:_type_4' 'ORGANIZATION_TYPE_Industry:_type_5'\\n 'ORGANIZATION_TYPE_Industry:_type_6' 'ORGANIZATION_TYPE_Industry:_type_7'\\n 'ORGANIZATION_TYPE_Industry:_type_8' 'ORGANIZATION_TYPE_Industry:_type_9'\\n 'ORGANIZATION_TYPE_Legal_Services'\\n 'ORGANIZATION_TYPE_Security_Ministries' 'ORGANIZATION_TYPE_Trade:_type_1'\\n 'ORGANIZATION_TYPE_Trade:_type_2' 'ORGANIZATION_TYPE_Trade:_type_3'\\n 'ORGANIZATION_TYPE_Trade:_type_4' 'ORGANIZATION_TYPE_Trade:_type_5'\\n 'ORGANIZATION_TYPE_Trade:_type_6' 'ORGANIZATION_TYPE_Trade:_type_7'\\n 'ORGANIZATION_TYPE_Transport:_type_1'\\n 'ORGANIZATION_TYPE_Transport:_type_2'\\n 'ORGANIZATION_TYPE_Transport:_type_3'\\n 'ORGANIZATION_TYPE_Transport:_type_4' 'WALLSMATERIAL_MODE_Stone,_brick'\\n 'BURO_CREDIT_ACTIVE_Bad_debt_MEAN' 'BURO_CREDIT_CURRENCY_currency_1_MEAN'\\n 'BURO_CREDIT_CURRENCY_currency_2_MEAN'\\n 'BURO_CREDIT_CURRENCY_currency_4_MEAN'\\n 'BURO_CREDIT_TYPE_Another_type_of_loan_MEAN'\\n 'BURO_CREDIT_TYPE_Car_loan_MEAN' 'BURO_CREDIT_TYPE_Consumer_credit_MEAN'\\n 'BURO_CREDIT_TYPE_Credit_card_MEAN'\\n 'BURO_CREDIT_TYPE_Loan_for_business_development_MEAN'\\n 'BURO_CREDIT_TYPE_Loan_for_working_capital_replenishment_MEAN'\\n 'BURO_CREDIT_TYPE_Real_estate_loan_MEAN'\\n 'BURO_CREDIT_TYPE_Unknown_type_of_loan_MEAN'\\n 'PREV_NAME_CONTRACT_TYPE_Cash_loans_MEAN'\\n 'PREV_NAME_CONTRACT_TYPE_Consumer_loans_MEAN'\\n 'PREV_NAME_CONTRACT_TYPE_Revolving_loans_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Building_a_house_or_an_annex_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Business_development_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Buying_a_garage_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Buying_a_holiday_home_/_land_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Buying_a_home_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Buying_a_new_car_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Buying_a_used_car_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Car_repairs_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Everyday_expenses_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Gasification_/_water_supply_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Payments_on_other_loans_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Purchase_of_electronic_equipment_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Urgent_needs_MEAN'\\n 'PREV_NAME_CASH_LOAN_PURPOSE_Wedding_/_gift_/_holiday_MEAN'\\n 'PREV_NAME_CONTRACT_STATUS_Unused_offer_MEAN'\\n 'PREV_NAME_PAYMENT_TYPE_Cash_through_the_bank_MEAN'\\n 'PREV_NAME_PAYMENT_TYPE_Cashless_from_the_account_of_the_employer_MEAN'\\n 'PREV_NAME_PAYMENT_TYPE_Non-cash_from_your_account_MEAN'\\n 'PREV_NAME_TYPE_SUITE_Group_of_people_MEAN'\\n 'PREV_NAME_TYPE_SUITE_Spouse,_partner_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Auto_Accessories_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Clothing_and_Accessories_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Construction_Materials_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Consumer_Electronics_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Direct_Sales_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Medical_Supplies_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Office_Appliances_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Photo_/_Cinema_Equipment_MEAN'\\n 'PREV_NAME_GOODS_CATEGORY_Sport_and_Leisure_MEAN'\\n 'PREV_CHANNEL_TYPE_AP+_(Cash_loan)_MEAN'\\n 'PREV_CHANNEL_TYPE_Car_dealer_MEAN'\\n 'PREV_CHANNEL_TYPE_Channel_of_corporate_sales_MEAN'\\n 'PREV_CHANNEL_TYPE_Contact_center_MEAN'\\n 'PREV_CHANNEL_TYPE_Credit_and_cash_offices_MEAN'\\n 'PREV_CHANNEL_TYPE_Regional_/_Local_MEAN'\\n 'PREV_NAME_SELLER_INDUSTRY_Auto_technology_MEAN'\\n 'PREV_NAME_SELLER_INDUSTRY_Consumer_electronics_MEAN'\\n 'PREV_NAME_SELLER_INDUSTRY_MLM_partners_MEAN'\\n 'PREV_PRODUCT_COMBINATION_Card_Street_MEAN'\\n 'PREV_PRODUCT_COMBINATION_Card_X-Sell_MEAN'\\n 'PREV_PRODUCT_COMBINATION_Cash_Street:_high_MEAN'\\n 'PREV_PRODUCT_COMBINATION_Cash_Street:_low_MEAN'\\n 'PREV_PRODUCT_COMBINATION_Cash_Street:_middle_MEAN'\\n 'PREV_PRODUCT_COMBINATION_Cash_X-Sell:_high_MEAN'\\n 'PREV_PRODUCT_COMBINATION_Cash_X-Sell:_low_MEAN'\\n 'PREV_PRODUCT_COMBINATION_Cash_X-Sell:_middle_MEAN'\\n 'PREV_PRODUCT_COMBINATION_POS_household_with_interest_MEAN'\\n 'PREV_PRODUCT_COMBINATION_POS_household_without_interest_MEAN'\\n 'PREV_PRODUCT_COMBINATION_POS_industry_with_interest_MEAN'\\n 'PREV_PRODUCT_COMBINATION_POS_industry_without_interest_MEAN'\\n 'PREV_PRODUCT_COMBINATION_POS_mobile_with_interest_MEAN'\\n 'PREV_PRODUCT_COMBINATION_POS_mobile_without_interest_MEAN'\\n 'PREV_PRODUCT_COMBINATION_POS_other_with_interest_MEAN'\\n 'PREV_PRODUCT_COMBINATION_POS_others_without_interest_MEAN'\\n 'POS_NAME_CONTRACT_STATUS_Returned_to_the_store_MEAN'\\n 'CC_NAME_CONTRACT_STATUS_Sent_proposal_MIN'\\n 'CC_NAME_CONTRACT_STATUS_Sent_proposal_MAX'\\n 'CC_NAME_CONTRACT_STATUS_Sent_proposal_MEAN'\\n 'CC_NAME_CONTRACT_STATUS_Sent_proposal_SUM'\\n 'CC_NAME_CONTRACT_STATUS_Sent_proposal_VAR'] not in index\""
     ]
    }
   ],
   "source": [
    "res_gp = gp_minimize(objective, space, n_calls=20,\n",
    "                     random_state=0, n_random_starts=10)\n",
    "\n",
    "\"Best score=%.4f\" % res_gp.fun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 221,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ORGANIZATION_TYPE_Industry:_type_7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "311"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_importance = feat_importance.groupby(['feature'])['importance'].mean()\n",
    "#avg_importance.sort_values(ascending=False)\n",
    "sum(avg_importance > 0.0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
